package com.eric.downloader;

import java.io.BufferedReader;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.eric.utils.UrlUtils;

public class SeoDownloader extends SimplePageDownloader {
			
	public SeoDownloader() {
		super();
	}
	
	//Download And Store Rules Into robotsRulesMap
	public HashMap<String, JSONArray> downloadRobotsRules(String url, HashMap<String, JSONArray> robotsRulesMap) throws MalformedURLException {
		
		download(UrlUtils.robotsUrl(url));
		String robotsTxt = html;
		BufferedReader br = new BufferedReader(new StringReader(robotsTxt));
		
		JSONObject agentRules = new JSONObject();;
		JSONArray disallow = new JSONArray();
		JSONArray allow = new JSONArray();
		JSONArray agentsJA = new JSONArray();
		
		String line;
		try{
			while ((line = br.readLine()) != null) {
				if (line.startsWith("#")) continue;
				
				else if (line.startsWith("User-agent")) {
					
					if (disallow.size() > 0 || allow.size() > 0) {
						agentRules.put("Disallow", disallow.clone());
						agentRules.put("Allow", allow.clone());
						agentsJA.add(agentRules.clone());
						
						
						agentRules = new JSONObject();
						disallow = new JSONArray();
						allow = new JSONArray();
//						System.out.println("Agent now:"+agentRules);
					}
					agentRules.put("User-agent", line.replaceAll("User-agent: ?", ""));
				}
				
				else if (line.startsWith("Disallow")) {
					disallow.add(line.replaceAll("Disallow: ?", ""));
				}
				
				else if (line.startsWith("Allow")) {
					allow.add(line.replaceAll("Allow: ?", ""));
				}
				
				else if (line.startsWith("Sitemap")) {
					agentRules.put("Sitemap", line.replaceAll("Sitemap: ?", ""));
				}
			}
			agentRules.put("Disallow", disallow.clone());
			agentRules.put("Allow", allow.clone());
			agentsJA.add(agentRules.clone());
		}catch (Exception e) {
			
		}
		robotsRulesMap.put(new URL(url).getHost(), agentsJA);
		
		System.out.println(robotsRulesMap);
		return robotsRulesMap;
	}
}
